#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2018-08-17 11:39:35
# @Author  : yangchaojun (YYChildren@gmail.com)
# @Link    : https://git.mingchao.com/yangchaojun
# @Version : 

from mtmodel.utils import pd_source, misc
from mtmodel.utils.config import CONFIG
from functools import partial
import math
import sys
import re
from tqdm import tqdm
tqdm.pandas()
from mtmodel.model import text_process
import logging
logging.basicConfig(level=logging.INFO)

def max_len_sentence(L):
    max_len = 0
    max_sentence = ''
    for s in L:
        m = len(s)
        if m > max_len:
            max_len = m
            max_sentence = s
    return max_sentence

def gen_game_keywords():
    sql = "select d.source,d.game_id,d.game_name,d.description,d.category from game_source.s_game_detail d WHERE d.source ='taptap'"
    df_detail = pd_source.from_sql(sql)
    df_detail['description'] = df_detail['description'].apply(text_process.clear_text)
    df_detail = df_detail[df_detail['description'].apply(text_process.is_chinese_sentence)].reset_index()
    df_detail = df_detail[['game_name', 'description']].groupby(['game_name']).agg(max_len_sentence).reset_index(level = ['game_name'])
    idf_dict={}
    for words in df_detail['description'].apply(text_process.tokenize):
        for word in set(words):
            idf_dict[word] = idf_dict.get(word, 0) + 1
    total_size = float(len(df_detail))
    idf_dict ={k: math.log(total_size / (v + 1)) for k,v in idf_dict.items()}
    idf_path=CONFIG['game_keywords']['idf_path']
    with open(idf_path, 'wb') as idf_file:
        content = "\n".join([ " ".join([k, str(v)]) for k,v in sorted(idf_dict.items())])
        idf_file.write(content.encode('utf-8'))
    text_process.idf_textrank.set_idf_path(idf_path)
    df_detail['keywords'] = df_detail['description'].progress_apply(text_process.extract_keywords, topK=10, withWeight=True)
    return df_detail['keywords']

def main():
    gen_game_keywords()

if __name__ == '__main__':
    main()